{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from functools import partial\n",
    "\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.linear_model import LinearRegression, BayesianRidge\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor, BaggingRegressor\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.model_selection import cross_val_score, RandomizedSearchCV\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from hyperopt import hp, fmin, tpe, Trials, STATUS_OK"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Загрузим датасет с жильем в Бостоне, разделим выборки на обучающую и тестовую, выполним стандартизацию."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = load_boston(return_X_y=True)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(X_train)\n",
    "X_train_std = scaler.transform(X_train)\n",
    "X_test_std = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Тестирование модели SVR\n",
    "\n",
    "Для каждой тестируемой модели будем выполнять байесовскую оптимизацию и случайный решетчатый поиск.\n",
    "\n",
    "Для байесовской оптимизации создадим функцию кросс-валидации."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_validation(params, estimator, X, y, cv=2):\n",
    "    \"\"\"\n",
    "    Кросс-валидация с текущими гиперпараметрами\n",
    "    \n",
    "    params : dict\n",
    "      Словарь гиперпараметров\n",
    "    estimator : sklearn model\n",
    "      Модель sklearn\n",
    "    X : matrix\n",
    "      Матрица признаков\n",
    "    y : вектор целевых значений   \n",
    "    \"\"\"\n",
    "    \n",
    "    estimator.set_params(**params)\n",
    "    \n",
    "    score = cross_val_score(estimator=estimator, X=X, y=y, cv=cv)\n",
    "    \n",
    "    return {'loss': -score.mean(), 'params': params, 'status': STATUS_OK}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_space = hp.choice(label='params', options=[\n",
    "    \n",
    "    {'kernel': hp.choice(label='criterion_1',  options=['linear']),\n",
    "     'C': hp.uniform(label='C_1', low=1e-12, high=10)},\n",
    "    \n",
    "    {'kernel': hp.choice(label='criterion_2',  options=['sigmoid']),\n",
    "     'C': hp.uniform(label='C_2', low=1e-12, high=10),\n",
    "     'coef0': hp.uniform(label='coef0_2', low=1e-12, high=10)},\n",
    "    \n",
    "    {'kernel': hp.choice(label='criterion_3',  options=['rbf']),\n",
    "     'C': hp.uniform(label='C_3', low=1e-12, high=10)},\n",
    "    \n",
    "    {'kernel': hp.choice(label='criterion_4',  options=['poly']),\n",
    "     'degree': hp.quniform(label='degree', low=3, high=8, q=1),\n",
    "     'C': hp.uniform(label='C_4', low=1e-12, high=2),\n",
    "     'coef0': hp.uniform(label='coef0_4', low=1e-12, high=10)},\n",
    "]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████| 50/50 [02:35<00:00,  3.11s/trial, best loss: -0.8259198885966708]\n"
     ]
    }
   ],
   "source": [
    "trials = Trials()\n",
    "svr_model = SVR()\n",
    "bayesian_search = fmin( \n",
    "  # функция для оптимизации  \n",
    "    fn=partial(cross_validation, estimator=svr_model, X=X_train_std, y=y_train, cv=5),\n",
    "  # пространство поиска гиперпараметров  \n",
    "    space=search_space,\n",
    "  # алгоритм поиска\n",
    "    algo=tpe.suggest,\n",
    "  # число итераций \n",
    "  # (можно ещё указать и время поиска) \n",
    "    max_evals=50,\n",
    "  # куда сохранять историю поиска\n",
    "    trials=trials,\n",
    "  # random state\n",
    "    rstate=np.random.RandomState(42),\n",
    "  # progressbar\n",
    "    show_progressbar=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVR, байесовская оптимизация\n",
      "\n",
      "Лучшая метрика R2: 0.8259198885966708\n",
      "Лучшие параматры:\n",
      "{'C_4': 0.9146782376276084, 'coef0_4': 8.118629376481968, 'criterion_4': 0, 'degree': 3.0, 'params': 3}\n"
     ]
    }
   ],
   "source": [
    "print('SVR, байесовская оптимизация\\n')\n",
    "print('Лучшая метрика R2:', -trials.best_trial['result']['loss'])\n",
    "print('Лучшие параматры:\\n', bayesian_search, sep='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В результате байесовской оптимизации params = 3 и criterion_4 = 0 означают, что было выбрано полиномиальное ядро.\n",
    "\n",
    "Теперь проверим какие результаты даст RandomizedSearchCV с аналогичными параметрами поиска."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = [\n",
    "    {'kernel': ['linear'],\n",
    "     'C': np.linspace(1e-12, 10, 1000)},\n",
    "    \n",
    "    {'kernel': ['sigmoid'],\n",
    "     'C': np.linspace(1e-12, 10, 1000),\n",
    "     'coef0': np.linspace(1e-12, 10, 1000)},\n",
    "    \n",
    "    {'kernel': ['rbf'],\n",
    "     'C': np.linspace(1e-12, 10, 1000)},\n",
    "    \n",
    "    {'kernel': ['poly'],\n",
    "     'degree': np.arange(2,9,1),\n",
    "     'C': np.linspace(1e-12, 10, 1000),\n",
    "     'coef0': np.linspace(1e-12, 10, 1000)},\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "rand_search = RandomizedSearchCV(estimator=svr_model, param_distributions=params, cv=5, n_iter=50,\n",
    "                                 random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomizedSearchCV(cv=5,\n",
       "                   estimator=SVR(C=7.861927986590262, coef0=7.241456456254678,\n",
       "                                 degree=4.0),\n",
       "                   n_iter=50,\n",
       "                   param_distributions=[{'C': array([1.00000000e-12, 1.00100100e-02, 2.00200200e-02, 3.00300300e-02,\n",
       "       4.00400400e-02, 5.00500501e-02, 6.00600601e-02, 7.00700701e-02,\n",
       "       8.00800801e-02, 9.00900901e-02, 1.00100100e-01, 1.10110110e-01,\n",
       "       1.20120120e-01, 1.30130130e-01, 1.4...\n",
       "       9.80980981e+00, 9.81981982e+00, 9.82982983e+00, 9.83983984e+00,\n",
       "       9.84984985e+00, 9.85985986e+00, 9.86986987e+00, 9.87987988e+00,\n",
       "       9.88988989e+00, 9.89989990e+00, 9.90990991e+00, 9.91991992e+00,\n",
       "       9.92992993e+00, 9.93993994e+00, 9.94994995e+00, 9.95995996e+00,\n",
       "       9.96996997e+00, 9.97997998e+00, 9.98998999e+00, 1.00000000e+01]),\n",
       "                                         'degree': array([2, 3, 4, 5, 6, 7, 8]),\n",
       "                                         'kernel': ['poly']}],\n",
       "                   random_state=42)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Подгонка может идти 10 минут и более\n",
    "rand_search.fit(X_train_std, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVR, случайный решетчатый поиск\n",
      "\n",
      "Лучшая метрика R2: 0.830476316410422\n",
      "Лучшие параматры:\n",
      "{'kernel': 'poly', 'degree': 2, 'coef0': 4.784784784785306, 'C': 8.98898898898909}\n"
     ]
    }
   ],
   "source": [
    "print('SVR, случайный решетчатый поиск\\n')\n",
    "print('Лучшая метрика R2:', rand_search.best_score_)\n",
    "print('Лучшие параматры:\\n', rand_search.best_params_, sep='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Случайный решетчатый поиск сработал немного лучше.\n",
    "\n",
    "## 2. Тестирование модели дерева принятий решений"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dectree_model = DecisionTreeRegressor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████| 500/500 [00:21<00:00, 23.20trial/s, best loss: -0.732071724762168]\n"
     ]
    }
   ],
   "source": [
    "search_space = {'criterion': hp.choice(label='criterion',  options=[\"mse\", \"friedman_mse\", \"mae\", \"poisson\"]),\n",
    "                'splitter': hp.choice(label='splitter',  options=[\"best\", \"random\"]),\n",
    "                'max_depth': hp.quniform(label='max_depth', low=1, high=1000, q=1),\n",
    "                'random_state': hp.choice(label='random_state',  options=[42])}\n",
    "\n",
    "trials = Trials()\n",
    "bayesian_search = fmin( \n",
    "    fn=partial(cross_validation, estimator=dectree_model, X=X_train_std, y=y_train, cv=5),\n",
    "    space=search_space,\n",
    "    algo=tpe.suggest,\n",
    "    max_evals=500,\n",
    "    trials=trials,\n",
    "    rstate=np.random.RandomState(42),\n",
    "    show_progressbar=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Дерево принятий решений, байесовская оптимизация\n",
      "\n",
      "Лучшая метрика R2: 0.732071724762168\n",
      "Лучшие параматры:\n",
      "{'criterion': 0, 'max_depth': 18.0, 'random_state': 0, 'splitter': 0}\n"
     ]
    }
   ],
   "source": [
    "print('Дерево принятий решений, байесовская оптимизация\\n')\n",
    "print('Лучшая метрика R2:', -trials.best_trial['result']['loss'])\n",
    "print('Лучшие параматры:\\n', bayesian_search, sep='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В данном случае criterion = 0 обозначает criterion = mse, а splitter = 0 обозначает splitter = best.\n",
    "\n",
    "Теперь проверим какие результаты даст RandomizedSearchCV с аналогичными параметрами поиска."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Дерево принятий решений, cлучайный решетчатый поиск\n",
      "\n",
      "Лучшая метрика R2: 0.7308587061213911\n",
      "Лучшие параматры:\n",
      "{'splitter': 'best', 'random_state': 42, 'max_depth': 354, 'criterion': 'mse'}\n"
     ]
    }
   ],
   "source": [
    "params = {'criterion': [\"mse\", \"friedman_mse\", \"mae\", \"poisson\"],\n",
    "          'splitter': [\"best\", \"random\"],\n",
    "          'max_depth': range(1, 1000),\n",
    "          'random_state': [42]}\n",
    "\n",
    "rand_search = RandomizedSearchCV(estimator=dectree_model, param_distributions=params, cv=5, n_iter=500,\n",
    "                                 random_state=42)\n",
    "rand_search.fit(X_train_std, y_train)\n",
    "\n",
    "print('Дерево принятий решений, cлучайный решетчатый поиск\\n')\n",
    "print('Лучшая метрика R2:', rand_search.best_score_)\n",
    "print('Лучшие параматры:\\n', rand_search.best_params_, sep='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В этот раз байесовская оптимизация сработала немного лучше.\n",
    "\n",
    "## 3. Тестирование модели случайного леса деревьев\n",
    "\n",
    "Под случайный лес необходимо переделать функцию кросс-валидации, у библиотеки hyperopt не нашел возможности задать количество n_estimators в целых числах с исключением нуля. Только целые числа в формате float через hp.quniform с q = 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████| 50/50 [01:41<00:00,  2.04s/trial, best loss: -0.8140513014053363]\n"
     ]
    }
   ],
   "source": [
    "def cross_validation_randforest(params, estimator, X, y, cv=2):\n",
    "    params['n_estimators'] = int(params['n_estimators'])\n",
    "    estimator.set_params(**params)\n",
    "    score = cross_val_score(estimator=estimator, X=X, y=y, cv=cv)\n",
    "    return {'loss': -score.mean(), 'params': params, 'status': STATUS_OK}\n",
    "\n",
    "\n",
    "randree_model = RandomForestRegressor()\n",
    "\n",
    "search_space = {'criterion': hp.choice(label='criterion',  options=[\"mse\", \"mae\"]),\n",
    "                'n_estimators': hp.quniform(label='n_estimators', low=10, high=200, q=1),\n",
    "                'random_state': hp.choice(label='random_state',  options=[42])}\n",
    "\n",
    "trials = Trials()\n",
    "bayesian_search = fmin( \n",
    "    fn=partial(cross_validation_randforest, estimator=randree_model, X=X_train_std, y=y_train, cv=5),\n",
    "    space=search_space,\n",
    "    algo=tpe.suggest,\n",
    "    max_evals=50,\n",
    "    trials=trials,\n",
    "    rstate=np.random.RandomState(42),\n",
    "    show_progressbar=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Случайный лес, байесовская оптимизация\n",
      "\n",
      "Лучшая метрика R2: 0.8140513014053363\n",
      "Лучшие параматры:\n",
      "{'criterion': 0, 'n_estimators': 113.0, 'random_state': 0}\n"
     ]
    }
   ],
   "source": [
    "print('Случайный лес, байесовская оптимизация\\n')\n",
    "print('Лучшая метрика R2:', -trials.best_trial['result']['loss'])\n",
    "print('Лучшие параматры:\\n', bayesian_search, sep='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Случайный лес, cлучайный решетчатый поиск\n",
      "\n",
      "Лучшая метрика R2: 0.8137525844097379\n",
      "Лучшие параматры:\n",
      "{'random_state': 42, 'n_estimators': 136, 'criterion': 'mse'}\n"
     ]
    }
   ],
   "source": [
    "params = {'criterion': [\"mse\", \"mae\"],\n",
    "          'n_estimators': range(10, 200),\n",
    "          'random_state': [42]}\n",
    "\n",
    "rand_search = RandomizedSearchCV(estimator=randree_model, param_distributions=params, cv=5, n_iter=50,\n",
    "                                 random_state=42)\n",
    "rand_search.fit(X_train_std, y_train)\n",
    "\n",
    "print('Случайный лес, cлучайный решетчатый поиск\\n')\n",
    "print('Лучшая метрика R2:', rand_search.best_score_)\n",
    "print('Лучшие параматры:\\n', rand_search.best_params_, sep='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Для случайного леса байесовская оптимизация дала лучший результат, но незначительно.\n",
    "\n",
    "## 4. Тестирование модели BayesianRidge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "bayesianridge_model = BayesianRidge()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████| 500/500 [00:11<00:00, 43.47trial/s, best loss: -0.690135701077619]\n"
     ]
    }
   ],
   "source": [
    "search_space = {'alpha_1': hp.uniform(label='alpha_1', low=1e-12, high=1e-3),\n",
    "                'alpha_2': hp.uniform(label='alpha_2', low=1e-12, high=1e-3),\n",
    "                'lambda_1': hp.uniform(label='lambda_1', low=1e-12, high=1e-3),\n",
    "                'lambda_2': hp.uniform(label='lambda_2', low=1e-12, high=1e-3)}\n",
    "\n",
    "trials = Trials()\n",
    "bayesian_search = fmin( \n",
    "    fn=partial(cross_validation, estimator=bayesianridge_model, X=X_train_std, y=y_train, cv=5),\n",
    "    space=search_space,\n",
    "    algo=tpe.suggest,\n",
    "    max_evals=500,\n",
    "    trials=trials,\n",
    "    rstate=np.random.RandomState(42),\n",
    "    show_progressbar=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BayesianRidge, байесовская оптимизация\n",
      "\n",
      "Лучшая метрика R2: 0.690135701077619\n",
      "Лучшие параматры:\n",
      "{'alpha_1': 2.207406862020356e-05, 'alpha_2': 0.00012170803513291944, 'lambda_1': 0.0009989520424490438, 'lambda_2': 1.438476970661669e-06}\n"
     ]
    }
   ],
   "source": [
    "print('BayesianRidge, байесовская оптимизация\\n')\n",
    "print('Лучшая метрика R2:', -trials.best_trial['result']['loss'])\n",
    "print('Лучшие параматры:\\n', bayesian_search, sep='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BayesianRidge, случайный решетчатый поиск\n",
      "\n",
      "Лучшая метрика R2: 0.6901564790022852\n",
      "Лучшие параматры:\n",
      "{'lambda_2': 0.003030303031272727, 'lambda_1': 0.09090909090918181, 'alpha_2': 0.0040404040413636365, 'alpha_1': 0.03232323232390909}\n"
     ]
    }
   ],
   "source": [
    "params = {'alpha_1': np.linspace(1e-12, 1e-1, 100),\n",
    "          'alpha_2': np.linspace(1e-12, 1e-1, 100),\n",
    "          'lambda_1': np.linspace(1e-12, 1e-1, 100),\n",
    "          'lambda_2': np.linspace(1e-12, 1e-1, 100)}\n",
    "\n",
    "rand_search = RandomizedSearchCV(estimator=bayesianridge_model, param_distributions=params, cv=5, n_iter=50,\n",
    "                                 random_state=42)\n",
    "rand_search.fit(X_train_std, y_train)\n",
    "\n",
    "print('BayesianRidge, случайный решетчатый поиск\\n')\n",
    "print('Лучшая метрика R2:', rand_search.best_score_)\n",
    "print('Лучшие параматры:\\n', rand_search.best_params_, sep='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Тестирование ансамблевой модели BaggingRegressor на основе деревьев принятий решений"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████| 50/50 [00:11<00:00,  4.51trial/s, best loss: -0.8146614016618127]\n"
     ]
    }
   ],
   "source": [
    "bagging = BaggingRegressor()\n",
    "\n",
    "search_space = {'max_samples': hp.uniform(label='max_samples', low=0.1, high=1),\n",
    "                'max_features': hp.uniform(label='max_features ', low=0.1, high=1),\n",
    "                'n_estimators': hp.quniform(label='n_estimators', low=5, high=30, q=1),\n",
    "                'random_state': hp.choice(label='random_state',  options=[42])}\n",
    "\n",
    "trials = Trials()\n",
    "bayesian_search = fmin( \n",
    "    fn=partial(cross_validation_randforest, estimator=bagging, X=X_train_std, y=y_train, cv=5),\n",
    "    space=search_space,\n",
    "    algo=tpe.suggest,\n",
    "    max_evals=50,\n",
    "    trials=trials,\n",
    "    rstate=np.random.RandomState(42),\n",
    "    show_progressbar=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Бэггинг из деревьев решений, байесовская оптимизация\n",
      "\n",
      "Лучшая метрика R2: 0.8146614016618127\n",
      "Лучшие параматры:\n",
      "{'max_features ': 0.9737584269557823, 'max_samples': 0.7655632847531592, 'n_estimators': 30.0, 'random_state': 0}\n"
     ]
    }
   ],
   "source": [
    "print('Бэггинг из деревьев решений, байесовская оптимизация\\n')\n",
    "print('Лучшая метрика R2:', -trials.best_trial['result']['loss'])\n",
    "print('Лучшие параматры:\\n', bayesian_search, sep='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Бэггинг из деревьев решений, случайный решетчатый поиск\n",
      "\n",
      "Лучшая метрика R2: 0.7982897005130849\n",
      "Лучшие параматры:\n",
      "{'random_state': 42, 'n_estimators': 6, 'max_samples': 0.709090909090909, 'max_features': 0.8727272727272727}\n"
     ]
    }
   ],
   "source": [
    "params = {'max_samples': np.linspace(0.1, 1, 100),\n",
    "          'max_features': np.linspace(0.1, 1, 100),\n",
    "          'n_estimators': range(5, 30),\n",
    "          'random_state': [42]}\n",
    "\n",
    "rand_search = RandomizedSearchCV(estimator=bagging, param_distributions=params, cv=5, n_iter=50,\n",
    "                                 random_state=42)\n",
    "rand_search.fit(X_train_std, y_train)\n",
    "\n",
    "print('Бэггинг из деревьев решений, случайный решетчатый поиск\\n')\n",
    "print('Лучшая метрика R2:', rand_search.best_score_)\n",
    "print('Лучшие параматры:\\n', rand_search.best_params_, sep='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "В этом тесте байесовская оптимизация дала более лучший результат."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
